{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/theamrzaki/text_summurization_abstractive_methods/blob/master/Model_2_seq2seq_using_lib.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HdMyi2D4-6YC"
   },
   "source": [
    "# **Model 2 seq2seq for text Summurization using seq2seq lib**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IzJv9UUti_og"
   },
   "source": [
    "### Intro\n",
    "This is a modification to https://github.com/dongjun-Lee/text-summarization-tensorflow \n",
    "I am builging it in a notebook envronment to be able to easily integrate with colab\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6lSefB0pn9Gq"
   },
   "source": [
    "## Helpers (Googel Drive , Utilits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xO0UtYGfCuU-"
   },
   "source": [
    "### Google Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mGpMpizWCwK1"
   },
   "outputs": [],
   "source": [
    "#working google drive ya RAB\n",
    "#https://stackoverflow.com/questions/52385655/unable-to-locate-package-google-drive-ocamlfuse-suddenly-stopped-working\n",
    "\n",
    "\n",
    "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
    "!wget https://launchpad.net/~alessandro-strada/+archive/ubuntu/google-drive-ocamlfuse-beta/+build/15331130/+files/google-drive-ocamlfuse_0.7.0-0ubuntu1_amd64.deb\n",
    "!dpkg -i google-drive-ocamlfuse_0.7.0-0ubuntu1_amd64.deb\n",
    "!apt-get install -f\n",
    "!apt-get -y install -qq fuse\n",
    "from google.colab import auth\n",
    "auth.authenticate_user()\n",
    "from oauth2client.client import GoogleCredentials\n",
    "creds = GoogleCredentials.get_application_default()\n",
    "import getpass\n",
    "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
    "vcode = getpass.getpass()\n",
    "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}\n",
    "\n",
    "!mkdir -p drive\n",
    "!google-drive-ocamlfuse drive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2pNpy7UgCx9m"
   },
   "source": [
    "### Utilits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X0VWHFAnoDL4"
   },
   "source": [
    "https://github.com/dongjun-Lee/text-summarization-tensorflow/blob/master/utils.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hf4b16KopBx7"
   },
   "outputs": [],
   "source": [
    "!pip install gensim\n",
    "!pip install wget\n",
    "  \n",
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lA1tCj2mn_2x"
   },
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "import re\n",
    "import collections\n",
    "import pickle\n",
    "import numpy as np\n",
    "from gensim.models.keyedvectors import KeyedVectors\n",
    "from gensim.test.utils import get_tmpfile\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "\n",
    "default_path = \"drive/Colab Notebooks/Model 2/\"\n",
    "\n",
    "train_article_path = default_path + \"sumdata/train/train.article.txt\"\n",
    "train_title_path   = default_path + \"sumdata/train/train.title.txt\"\n",
    "valid_article_path = default_path + \"sumdata/train/valid.article.filter.txt\"\n",
    "valid_title_path   = default_path + \"sumdata/train/valid.title.filter.txt\"\n",
    "\n",
    "#valid_article_path = default_path + \"sumdata/DUC2003/input.txt\"\n",
    "#valid_title_path   = default_path + \"sumdata/DUC2003/task1_ref0.txt\"\n",
    "\n",
    "def clean_str(sentence):\n",
    "    sentence = re.sub(\"[#.]+\", \"#\", sentence)\n",
    "    return sentence\n",
    "\n",
    "\n",
    "def get_text_list(data_path, toy):\n",
    "    with open (data_path, \"r\", encoding=\"utf-8\") as f:\n",
    "        if not toy:\n",
    "            return [clean_str(x.strip()) for x in f.readlines()][:200000]\n",
    "        else:\n",
    "            return [clean_str(x.strip()) for x in f.readlines()][:50]\n",
    "\n",
    "\n",
    "def build_dict(step, toy=False):\n",
    "    if step == \"train\":\n",
    "        train_article_list = get_text_list(train_article_path, toy)\n",
    "        train_title_list = get_text_list(train_title_path, toy)\n",
    "\n",
    "        words = list()\n",
    "        for sentence in train_article_list + train_title_list:\n",
    "            for word in word_tokenize(sentence):\n",
    "                words.append(word)\n",
    "\n",
    "        word_counter = collections.Counter(words).most_common()\n",
    "        word_dict = dict()\n",
    "        word_dict[\"<padding>\"] = 0\n",
    "        word_dict[\"<unk>\"] = 1\n",
    "        word_dict[\"<s>\"] = 2\n",
    "        word_dict[\"</s>\"] = 3\n",
    "        for word, _ in word_counter:\n",
    "            word_dict[word] = len(word_dict)\n",
    "\n",
    "        with open(default_path + \"word_dict.pickle\", \"wb\") as f:\n",
    "            pickle.dump(word_dict, f)\n",
    "\n",
    "    elif step == \"valid\":\n",
    "        with open(default_path + \"word_dict.pickle\", \"rb\") as f:\n",
    "            word_dict = pickle.load(f)\n",
    "\n",
    "    reversed_dict = dict(zip(word_dict.values(), word_dict.keys()))\n",
    "\n",
    "    article_max_len = 50\n",
    "    summary_max_len = 15\n",
    "\n",
    "    return word_dict, reversed_dict, article_max_len, summary_max_len\n",
    "\n",
    "\n",
    "def build_dataset(step, word_dict, article_max_len, summary_max_len, toy=False):\n",
    "    if step == \"train\":\n",
    "        article_list = get_text_list(train_article_path, toy)\n",
    "        title_list = get_text_list(train_title_path, toy)\n",
    "    elif step == \"valid\":\n",
    "        article_list = get_text_list(valid_article_path, toy)\n",
    "    else:\n",
    "        raise NotImplementedError\n",
    "\n",
    "    x = [word_tokenize(d) for d in article_list]\n",
    "    x = [[word_dict.get(w, word_dict[\"<unk>\"]) for w in d] for d in x]\n",
    "    x = [d[:article_max_len] for d in x]\n",
    "    x = [d + (article_max_len - len(d)) * [word_dict[\"<padding>\"]] for d in x]\n",
    "    \n",
    "    if step == \"valid\":\n",
    "        return x\n",
    "    else:        \n",
    "        y = [word_tokenize(d) for d in title_list]\n",
    "        y = [[word_dict.get(w, word_dict[\"<unk>\"]) for w in d] for d in y]\n",
    "        y = [d[:(summary_max_len - 1)] for d in y]\n",
    "        return x, y\n",
    "\n",
    "\n",
    "def batch_iter(inputs, outputs, batch_size, num_epochs):\n",
    "    inputs = np.array(inputs)\n",
    "    outputs = np.array(outputs)\n",
    "\n",
    "    num_batches_per_epoch = (len(inputs) - 1) // batch_size + 1\n",
    "    for epoch in range(num_epochs):\n",
    "        for batch_num in range(num_batches_per_epoch):\n",
    "            start_index = batch_num * batch_size\n",
    "            end_index = min((batch_num + 1) * batch_size, len(inputs))\n",
    "            yield inputs[start_index:end_index], outputs[start_index:end_index]\n",
    "\n",
    "\n",
    "def get_init_embedding(reversed_dict, embedding_size):\n",
    "    #glove_file = default_path + \"glove/glove.6B.300d.txt\"\n",
    "    #word2vec_file = get_tmpfile(default_path + \"word2vec_format.vec\")\n",
    "    #glove2word2vec(glove_file, word2vec_file)\n",
    "    print(\"Loading Glove vectors...\")\n",
    "    #word_vectors = KeyedVectors.load_word2vec_format(word2vec_file)\n",
    "\n",
    "    with open( default_path + \"glove/model_glove_300.pkl\", 'rb') as handle:\n",
    "        word_vectors = pickle.load(handle)\n",
    "        \n",
    "    word_vec_list = list()\n",
    "    for _, word in sorted(reversed_dict.items()):\n",
    "        try:\n",
    "            word_vec = word_vectors.word_vec(word)\n",
    "        except KeyError:\n",
    "            word_vec = np.zeros([embedding_size], dtype=np.float32)\n",
    "\n",
    "        word_vec_list.append(word_vec)\n",
    "\n",
    "    # Assign random vector to <s>, </s> token\n",
    "    word_vec_list[2] = np.random.normal(0, 1, embedding_size)\n",
    "    word_vec_list[3] = np.random.normal(0, 1, embedding_size)\n",
    "\n",
    "    return np.array(word_vec_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qI3AJND0mb0r"
   },
   "source": [
    "## Prepare Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X1B9ZgJqmsni"
   },
   "source": [
    "\n",
    "https://github.com/dongjun-Lee/text-summarization-tensorflow/blob/master/prep_data.py\n",
    "\n",
    "1.   Word Embedding : Used [Glove pre-trained vectors](https://nlp.stanford.edu/projects/glove/ ) to initialize word embedding.  \n",
    "2.   Dataset :  Dataset is available at [harvardnlp/sent-summary](https://github.com/harvardnlp/sent-summary). Locate the summary.tar.gz file in project root directory.   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g2dpVeOqnJAh"
   },
   "outputs": [],
   "source": [
    "import wget\n",
    "import os\n",
    "import tarfile\n",
    "import gzip\n",
    "import zipfile\n",
    "import argparse\n",
    "\n",
    "\n",
    "#parser = argparse.ArgumentParser()\n",
    "#parser.add_argument(\"--glove\", action=\"store_true\")\n",
    "#args = parser.parse_args()\n",
    "\n",
    "# Extract data file\n",
    "#with tarfile.open(default_path + \"sumdata/train/summary.tar.gz\", \"r:gz\") as tar:\n",
    "#    tar.extractall()\n",
    "\n",
    "with gzip.open(default_path + \"sumdata/train/train.article.txt.gz\", \"rb\") as gz:\n",
    "    with open(default_path + \"sumdata/train/train.article.txt\", \"wb\") as out:\n",
    "        out.write(gz.read())\n",
    "\n",
    "with gzip.open(default_path + \"sumdata/train/train.title.txt.gz\", \"rb\") as gz:\n",
    "    with open(default_path + \"sumdata/train/train.title.txt\", \"wb\") as out:\n",
    "        out.write(gz.read())\n",
    "\n",
    "        \n",
    "#if args.glove:\n",
    "#    glove_dir = \"glove\"\n",
    "#    glove_url = \"https://nlp.stanford.edu/data/wordvecs/glove.42B.300d.zip\"\n",
    "#\n",
    "#    if not os.path.exists(glove_dir):\n",
    "#        os.mkdir(glove_dir)\n",
    "#\n",
    "#    # Download glove vector\n",
    "#    wget.download(glove_url, out=glove_dir)\n",
    "#\n",
    "#    # Extract glove file\n",
    "#    with zipfile.ZipFile(os.path.join(\"glove\", \"glove.42B.300d.zip\"), \"r\") as z:\n",
    "#        z.extractall(glove_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "F_r1e_f2nCKK"
   },
   "source": [
    "##Model:\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MABp2aWMoNKh"
   },
   "source": [
    "Encoder-Decoder model with attention mechanism.\n",
    "https://github.com/dongjun-Lee/text-summarization-tensorflow/blob/master/model.py\n",
    "\n",
    "1.   Encoder : Used LSTM cell with stack_bidirectional_dynamic_rnn.\n",
    "2.   Decoder : Used LSTM BasicDecoder for training, and BeamSearchDecoder for inference.\n",
    "3.   Attention Mechanism : Used BahdanauAttention with weight normalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GKHwTTZg5eJa"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "#from utils import get_init_embedding\n",
    "\n",
    "\n",
    "class Model(object):\n",
    "    def __init__(self, reversed_dict, article_max_len, summary_max_len, args, forward_only=False):\n",
    "        self.vocabulary_size = len(reversed_dict)\n",
    "        self.embedding_size = args.embedding_size\n",
    "        self.num_hidden = args.num_hidden\n",
    "        self.num_layers = args.num_layers\n",
    "        self.learning_rate = args.learning_rate\n",
    "        self.beam_width = args.beam_width\n",
    "        if not forward_only:\n",
    "            self.keep_prob = args.keep_prob\n",
    "        else:\n",
    "            self.keep_prob = 1.0\n",
    "        self.cell = tf.nn.rnn_cell.BasicLSTMCell\n",
    "        with tf.variable_scope(\"decoder/projection\"):\n",
    "            self.projection_layer = tf.layers.Dense(self.vocabulary_size, use_bias=False)\n",
    "\n",
    "        self.batch_size = tf.placeholder(tf.int32, (), name=\"batch_size\")\n",
    "        self.X = tf.placeholder(tf.int32, [None, article_max_len])\n",
    "        self.X_len = tf.placeholder(tf.int32, [None])\n",
    "        self.decoder_input = tf.placeholder(tf.int32, [None, summary_max_len])\n",
    "        self.decoder_len = tf.placeholder(tf.int32, [None])\n",
    "        self.decoder_target = tf.placeholder(tf.int32, [None, summary_max_len])\n",
    "        self.global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "        with tf.name_scope(\"embedding\"):\n",
    "            if not forward_only and args.glove:\n",
    "                init_embeddings = tf.constant(get_init_embedding(reversed_dict, self.embedding_size), dtype=tf.float32)\n",
    "            else:\n",
    "                init_embeddings = tf.random_uniform([self.vocabulary_size, self.embedding_size], -1.0, 1.0)\n",
    "            self.embeddings = tf.get_variable(\"embeddings\", initializer=init_embeddings)\n",
    "            self.encoder_emb_inp = tf.transpose(tf.nn.embedding_lookup(self.embeddings, self.X), perm=[1, 0, 2])\n",
    "            self.decoder_emb_inp = tf.transpose(tf.nn.embedding_lookup(self.embeddings, self.decoder_input), perm=[1, 0, 2])\n",
    "\n",
    "        with tf.name_scope(\"encoder\"):\n",
    "            fw_cells = [self.cell(self.num_hidden) for _ in range(self.num_layers)]\n",
    "            bw_cells = [self.cell(self.num_hidden) for _ in range(self.num_layers)]\n",
    "            fw_cells = [rnn.DropoutWrapper(cell) for cell in fw_cells]\n",
    "            bw_cells = [rnn.DropoutWrapper(cell) for cell in bw_cells]\n",
    "\n",
    "            encoder_outputs, encoder_state_fw, encoder_state_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(\n",
    "                fw_cells, bw_cells, self.encoder_emb_inp,\n",
    "                sequence_length=self.X_len, time_major=True, dtype=tf.float32)\n",
    "            self.encoder_output = tf.concat(encoder_outputs, 2)\n",
    "            encoder_state_c = tf.concat((encoder_state_fw[0].c, encoder_state_bw[0].c), 1)\n",
    "            encoder_state_h = tf.concat((encoder_state_fw[0].h, encoder_state_bw[0].h), 1)\n",
    "            self.encoder_state = rnn.LSTMStateTuple(c=encoder_state_c, h=encoder_state_h)\n",
    "\n",
    "        with tf.name_scope(\"decoder\"), tf.variable_scope(\"decoder\") as decoder_scope:\n",
    "            decoder_cell = self.cell(self.num_hidden * 2)\n",
    "\n",
    "            if not forward_only:\n",
    "                attention_states = tf.transpose(self.encoder_output, [1, 0, 2])\n",
    "                attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                    self.num_hidden * 2, attention_states, memory_sequence_length=self.X_len, normalize=True)\n",
    "                decoder_cell = tf.contrib.seq2seq.AttentionWrapper(decoder_cell, attention_mechanism,\n",
    "                                                                   attention_layer_size=self.num_hidden * 2)\n",
    "                initial_state = decoder_cell.zero_state(dtype=tf.float32, batch_size=self.batch_size)\n",
    "                initial_state = initial_state.clone(cell_state=self.encoder_state)\n",
    "                helper = tf.contrib.seq2seq.TrainingHelper(self.decoder_emb_inp, self.decoder_len, time_major=True)\n",
    "                decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell, helper, initial_state)\n",
    "                outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder, output_time_major=True, scope=decoder_scope)\n",
    "                self.decoder_output = outputs.rnn_output\n",
    "                self.logits = tf.transpose(\n",
    "                    self.projection_layer(self.decoder_output), perm=[1, 0, 2])\n",
    "                self.logits_reshape = tf.concat(\n",
    "                    [self.logits, tf.zeros([self.batch_size, summary_max_len - tf.shape(self.logits)[1], self.vocabulary_size])], axis=1)\n",
    "            else:\n",
    "                tiled_encoder_output = tf.contrib.seq2seq.tile_batch(\n",
    "                    tf.transpose(self.encoder_output, perm=[1, 0, 2]), multiplier=self.beam_width)\n",
    "                tiled_encoder_final_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, multiplier=self.beam_width)\n",
    "                tiled_seq_len = tf.contrib.seq2seq.tile_batch(self.X_len, multiplier=self.beam_width)\n",
    "                attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                    self.num_hidden * 2, tiled_encoder_output, memory_sequence_length=tiled_seq_len, normalize=True)\n",
    "                decoder_cell = tf.contrib.seq2seq.AttentionWrapper(decoder_cell, attention_mechanism,\n",
    "                                                                   attention_layer_size=self.num_hidden * 2)\n",
    "                initial_state = decoder_cell.zero_state(dtype=tf.float32, batch_size=self.batch_size * self.beam_width)\n",
    "                initial_state = initial_state.clone(cell_state=tiled_encoder_final_state)\n",
    "                decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                    cell=decoder_cell,\n",
    "                    embedding=self.embeddings,\n",
    "                    start_tokens=tf.fill([self.batch_size], tf.constant(2)),\n",
    "                    end_token=tf.constant(3),\n",
    "                    initial_state=initial_state,\n",
    "                    beam_width=self.beam_width,\n",
    "                    output_layer=self.projection_layer\n",
    "                )\n",
    "                outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder, output_time_major=True, maximum_iterations=summary_max_len, scope=decoder_scope)\n",
    "                self.prediction = tf.transpose(outputs.predicted_ids, perm=[1, 2, 0])\n",
    "\n",
    "        with tf.name_scope(\"loss\"):\n",
    "            if not forward_only:\n",
    "                crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                    logits=self.logits_reshape, labels=self.decoder_target)\n",
    "                weights = tf.sequence_mask(self.decoder_len, summary_max_len, dtype=tf.float32)\n",
    "                self.loss = tf.reduce_sum(crossent * weights / tf.to_float(self.batch_size))\n",
    "\n",
    "                params = tf.trainable_variables()\n",
    "                gradients = tf.gradients(self.loss, params)\n",
    "                clipped_gradients, _ = tf.clip_by_global_norm(gradients, 5.0)\n",
    "                optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "                self.update = optimizer.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x_xON6b4nS8x"
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ak3tIhETnaIJ"
   },
   "source": [
    "https://github.com/dongjun-Lee/text-summarization-tensorflow/blob/master/train.py\n",
    "\n",
    "We used sumdata/train/train.article.txt and sumdata/train/train.title.txt for training data. To train the model, use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1882
    },
    "colab_type": "code",
    "id": "vq9ZA0hFnUZJ",
    "outputId": "fa10f381-4ceb-4d6e-8e82-7dab98daa5c7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building dictionary...\n",
      "Loading training dataset...\n",
      "Loading Glove vectors...\n",
      "WARNING:tensorflow:From <ipython-input-3-3e3c149b757b>:40: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is deprecated, please use tf.nn.rnn_cell.LSTMCell, which supports all the feature this cell currently has. Please replace the existing code with tf.nn.rnn_cell.LSTMCell(name='basic_lstm_cell').\n",
      "Continuing from previous trained model: drive/Colab Notebooks/Model 2/saved_model/model.ckpt-15625 ...\n",
      "INFO:tensorflow:Restoring parameters from drive/Colab Notebooks/Model 2/saved_model/model.ckpt-15625\n",
      "\n",
      "Iteration starts.\n",
      "Number of batches per epoch : 3125\n",
      "step 16000: loss = 17.542762756347656\n",
      "step 17000: loss = 17.148090362548828\n",
      "step 18000: loss = 16.15915298461914\n",
      " Epoch 6: Model is saved. Elapsed: 00:27:31.02 \n",
      "\n",
      "step 19000: loss = 16.067659378051758\n",
      "step 20000: loss = 13.77082633972168\n",
      "step 21000: loss = 9.985371589660645\n",
      " Epoch 7: Model is saved. Elapsed: 00:51:06.95 \n",
      "\n",
      "step 22000: loss = 9.060101509094238\n",
      "step 23000: loss = 10.273500442504883\n",
      "step 24000: loss = 8.802462577819824\n",
      "step 25000: loss = 14.184542655944824\n",
      " Epoch 8: Model is saved. Elapsed: 01:14:13.36 \n",
      "\n",
      "step 26000: loss = 15.260891914367676\n",
      "step 27000: loss = 9.954633712768555\n",
      "step 28000: loss = 12.338654518127441\n",
      " Epoch 9: Model is saved. Elapsed: 01:37:29.54 \n",
      "\n",
      "step 29000: loss = 11.25490665435791\n",
      "step 30000: loss = 16.303237915039062\n",
      "step 31000: loss = 8.08161735534668\n",
      " Epoch 10: Model is saved. Elapsed: 02:01:02.42 \n",
      "\n",
      "step 32000: loss = 13.465749740600586\n",
      "step 33000: loss = 7.876657009124756\n",
      "step 34000: loss = 8.820916175842285\n",
      " Epoch 11: Model is saved. Elapsed: 02:24:55.21 \n",
      "\n",
      "step 35000: loss = 7.746098518371582\n",
      "step 36000: loss = 7.245748519897461\n",
      "step 37000: loss = 5.104197025299072\n",
      " Epoch 12: Model is saved. Elapsed: 02:48:51.43 \n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-fc0e0a25e733>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    101\u001b[0m         }\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 103\u001b[0;31m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_feed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m1000\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1319\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1407\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1409\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import time\n",
    "start = time.perf_counter()\n",
    "import tensorflow as tf\n",
    "import argparse\n",
    "import pickle\n",
    "import os\n",
    "#from model import Model\n",
    "#from utils import build_dict, build_dataset, batch_iter\n",
    "\n",
    "# Uncomment next 2 lines to suppress error and Tensorflow info verbosity. Or change logging levels\n",
    "# tf.logging.set_verbosity(tf.logging.FATAL)\n",
    "# os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "\n",
    "#def add_arguments(parser):\n",
    "#    parser.add_argument(\"--num_hidden\", type=int, default=150, help=\"Network size.\")\n",
    "#    parser.add_argument(\"--num_layers\", type=int, default=2, help=\"Network depth.\")\n",
    "#    parser.add_argument(\"--beam_width\", type=int, default=10, help=\"Beam width for beam search decoder.\")\n",
    "#    parser.add_argument(\"--glove\", action=\"store_true\", help=\"Use glove as initial word embedding.\")\n",
    "#    parser.add_argument(\"--embedding_size\", type=int, default=300, help=\"Word embedding size.\")\n",
    "#\n",
    "#    parser.add_argument(\"--learning_rate\", type=float, default=1e-3, help=\"Learning rate.\")\n",
    "#    parser.add_argument(\"--batch_size\", type=int, default=64, help=\"Batch size.\")\n",
    "#    parser.add_argument(\"--num_epochs\", type=int, default=10, help=\"Number of epochs.\")\n",
    "#    parser.add_argument(\"--keep_prob\", type=float, default=0.8, help=\"Dropout keep prob.\")\n",
    "#\n",
    "#    parser.add_argument(\"--toy\", action=\"store_true\", help=\"Use only 50K samples of data\")\n",
    "#\n",
    "#    parser.add_argument(\"--with_model\", action=\"store_true\", help=\"Continue from previously saved model\")\n",
    "\n",
    "class args:\n",
    "    pass\n",
    "  \n",
    "args.num_hidden=150\n",
    "args.num_layers=2\n",
    "args.beam_width=10\n",
    "args.glove=\"store_true\"\n",
    "args.embedding_size=300\n",
    "\n",
    "args.learning_rate=1e-3\n",
    "args.batch_size=64\n",
    "args.num_epochs=10\n",
    "args.keep_prob = 0.8\n",
    "\n",
    "args.toy=False #\"store_true\"\n",
    "\n",
    "args.with_model=\"store_true\"\n",
    "\n",
    "\n",
    "#parser = argparse.ArgumentParser()\n",
    "#add_arguments(parser)\n",
    "#args = parser.parse_args()\n",
    "#with open(\"args.pickle\", \"wb\") as f:\n",
    "#    pickle.dump(args, f)\n",
    "\n",
    "if not os.path.exists(default_path + \"saved_model\"):\n",
    "    os.mkdir(default_path + \"saved_model\")\n",
    "else:\n",
    "    #if args.with_model:\n",
    "        old_model_checkpoint_path = open(default_path + 'saved_model/checkpoint', 'r')\n",
    "        old_model_checkpoint_path = \"\".join([default_path + \"saved_model/\",old_model_checkpoint_path.read().splitlines()[0].split('\"')[1] ])\n",
    "\n",
    "\n",
    "print(\"Building dictionary...\")\n",
    "word_dict, reversed_dict, article_max_len, summary_max_len = build_dict(\"train\", args.toy)\n",
    "print(\"Loading training dataset...\")\n",
    "train_x, train_y = build_dataset(\"train\", word_dict, article_max_len, summary_max_len, args.toy)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    model = Model(reversed_dict, article_max_len, summary_max_len, args)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    if 'old_model_checkpoint_path' in globals():\n",
    "        print(\"Continuing from previous trained model:\" , old_model_checkpoint_path , \"...\")\n",
    "        saver.restore(sess, old_model_checkpoint_path )\n",
    "\n",
    "    batches = batch_iter(train_x, train_y, args.batch_size, args.num_epochs)\n",
    "    num_batches_per_epoch = (len(train_x) - 1) // args.batch_size + 1\n",
    "\n",
    "    print(\"\\nIteration starts.\")\n",
    "    print(\"Number of batches per epoch :\", num_batches_per_epoch)\n",
    "    for batch_x, batch_y in batches:\n",
    "        batch_x_len = list(map(lambda x: len([y for y in x if y != 0]), batch_x))\n",
    "        batch_decoder_input = list(map(lambda x: [word_dict[\"<s>\"]] + list(x), batch_y))\n",
    "        batch_decoder_len = list(map(lambda x: len([y for y in x if y != 0]), batch_decoder_input))\n",
    "        batch_decoder_output = list(map(lambda x: list(x) + [word_dict[\"</s>\"]], batch_y))\n",
    "\n",
    "        batch_decoder_input = list(\n",
    "            map(lambda d: d + (summary_max_len - len(d)) * [word_dict[\"<padding>\"]], batch_decoder_input))\n",
    "        batch_decoder_output = list(\n",
    "            map(lambda d: d + (summary_max_len - len(d)) * [word_dict[\"<padding>\"]], batch_decoder_output))\n",
    "\n",
    "        train_feed_dict = {\n",
    "            model.batch_size: len(batch_x),\n",
    "            model.X: batch_x,\n",
    "            model.X_len: batch_x_len,\n",
    "            model.decoder_input: batch_decoder_input,\n",
    "            model.decoder_len: batch_decoder_len,\n",
    "            model.decoder_target: batch_decoder_output\n",
    "        }\n",
    "\n",
    "        _, step, loss = sess.run([model.update, model.global_step, model.loss], feed_dict=train_feed_dict)\n",
    "\n",
    "        if step % 1000 == 0:\n",
    "            print(\"step {0}: loss = {1}\".format(step, loss))\n",
    "\n",
    "        if step % num_batches_per_epoch == 0:\n",
    "            hours, rem = divmod(time.perf_counter() - start, 3600)\n",
    "            minutes, seconds = divmod(rem, 60)\n",
    "            saver.save(sess, default_path + \"saved_model/model.ckpt\", global_step=step)\n",
    "            print(\" Epoch {0}: Model is saved.\".format(step // num_batches_per_epoch),\n",
    "            \"Elapsed: {:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds) , \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TbbtwLe7njGS"
   },
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iCBIdwESnlhZ"
   },
   "source": [
    "https://github.com/dongjun-Lee/text-summarization-tensorflow/blob/master/test.py\n",
    "\n",
    "Generate summary of each article in sumdata/train/valid.article.filter.txt by"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 145
    },
    "colab_type": "code",
    "id": "1IUJ0dpon1Hi",
    "outputId": "4a20b076-a8b9-406d-9269-b01faf66e568"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dictionary...\n",
      "Loading validation dataset...\n",
      "Loading article and reference...\n",
      "Loading saved model...\n",
      "INFO:tensorflow:Restoring parameters from drive/Colab Notebooks/Model 2/saved_model/model.ckpt-37500\n",
      "Writing summaries to 'result.txt'...\n",
      "Summaries have been generated\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import pickle\n",
    "#from model import Model\n",
    "#from utils import build_dict, build_dataset, batch_iter\n",
    "\n",
    "\n",
    "#with open(\"args.pickle\", \"rb\") as f:\n",
    "#    args = pickle.load(f)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "class args:\n",
    "    pass\n",
    "  \n",
    "args.num_hidden=150\n",
    "args.num_layers=2\n",
    "args.beam_width=10\n",
    "args.glove=\"store_true\"\n",
    "args.embedding_size=300\n",
    "\n",
    "args.learning_rate=1e-3\n",
    "args.batch_size=64\n",
    "args.num_epochs=10\n",
    "args.keep_prob = 0.8\n",
    "\n",
    "args.toy=True\n",
    "\n",
    "args.with_model=\"store_true\"\n",
    "\n",
    "\n",
    "\n",
    "print(\"Loading dictionary...\")\n",
    "word_dict, reversed_dict, article_max_len, summary_max_len = build_dict(\"valid\", args.toy)\n",
    "print(\"Loading validation dataset...\")\n",
    "valid_x = build_dataset(\"valid\", word_dict, article_max_len, summary_max_len, args.toy)\n",
    "valid_x_len = [len([y for y in x if y != 0]) for x in valid_x]\n",
    "print(\"Loading article and reference...\")\n",
    "article = get_text_list(valid_article_path, args.toy)\n",
    "reference = get_text_list(valid_title_path, args.toy)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print(\"Loading saved model...\")\n",
    "    model = Model(reversed_dict, article_max_len, summary_max_len, args, forward_only=True)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    ckpt = tf.train.get_checkpoint_state(default_path + \"saved_model/\")\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "\n",
    "    batches = batch_iter(valid_x, [0] * len(valid_x), args.batch_size, 1)\n",
    "\n",
    "    print(\"Writing summaries to 'result.txt'...\")\n",
    "    for batch_x, _ in batches:\n",
    "        batch_x_len = [len([y for y in x if y != 0]) for x in batch_x]\n",
    "\n",
    "        valid_feed_dict = {\n",
    "            model.batch_size: len(batch_x),\n",
    "            model.X: batch_x,\n",
    "            model.X_len: batch_x_len,\n",
    "        }\n",
    "\n",
    "        prediction = sess.run(model.prediction, feed_dict=valid_feed_dict)\n",
    "        prediction_output = [[reversed_dict[y] for y in x] for x in prediction[:, 0, :]]\n",
    "        summary_array = []\n",
    "        with open(default_path + \"result.txt\", \"a\") as f:\n",
    "            for line in prediction_output:\n",
    "                summary = list()\n",
    "                for word in line:\n",
    "                    if word == \"</s>\":\n",
    "                        break\n",
    "                    if word not in summary:\n",
    "                        summary.append(word)\n",
    "                summary_array.append(\" \".join(summary))\n",
    "                #print(\" \".join(summary), file=f)\n",
    "\n",
    "    print('Summaries have been generated')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K-Kj9e5M9lBr"
   },
   "source": [
    "## Evaluate & write output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "svL3XWnXaTsW"
   },
   "source": [
    "for comparing (good resources)\n",
    "\n",
    "1.  [thunlp]( https://github.com/thunlp/TensorFlow-Summarization)     works with duc2003\n",
    "2.  [textsum](https://github.com/tensorflow/models/tree/master/research/textsum )   \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Tyi0SXZL0GnH"
   },
   "outputs": [],
   "source": [
    "!pip install sumeval\n",
    "!python -m spacy download en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x6nERCu29oPS"
   },
   "outputs": [],
   "source": [
    "#https://github.com/chakki-works/sumeval\n",
    "#https://github.com/Tian312/awesome-text-summarization\n",
    "\n",
    "from sumeval.metrics.rouge import RougeCalculator\n",
    "from sumeval.metrics.bleu import BLEUCalculator\n",
    "\n",
    "def eval_rouges(refrence_summary,model_summary):\n",
    "    #refrence_summary = \"tokyo shares close up #.## percent\"\n",
    "    #model_summary = \"tokyo stocks close up # percent to fresh record high\"\n",
    "\n",
    "    rouge = RougeCalculator(stopwords=True, lang=\"en\")\n",
    "\n",
    "    rouge_1 = rouge.rouge_n(\n",
    "                summary=model_summary,\n",
    "                references=refrence_summary,\n",
    "                n=1)\n",
    "\n",
    "    rouge_2 = rouge.rouge_n(\n",
    "                summary=model_summary,\n",
    "                references=[refrence_summary],\n",
    "                n=2)\n",
    "    \n",
    "    rouge_l = rouge.rouge_l(\n",
    "                summary=model_summary,\n",
    "                references=[refrence_summary])\n",
    "    \n",
    "    # You need spaCy to calculate ROUGE-BE\n",
    "    \n",
    "    rouge_be = rouge.rouge_be(\n",
    "                summary=model_summary,\n",
    "                references=[refrence_summary])\n",
    "\n",
    "    bleu = BLEUCalculator()\n",
    "    bleu_score = bleu.bleu( summary=model_summary,\n",
    "                        references=[refrence_summary])\n",
    "\n",
    "    #print(\"ROUGE-1: {}, ROUGE-2: {}, ROUGE-L: {}, ROUGE-BE: {}\".format(\n",
    "    #    rouge_1, rouge_2, rouge_l, rouge_be\n",
    "    #).replace(\", \", \"\\n\"))\n",
    "    \n",
    "    return rouge_1, rouge_2,rouge_l,rouge_be,bleu_score\n",
    "  \n",
    "#rouge_1, rouge_2,rouge_l,rouge_be = eval_rouges( \"tokyo shares close up #.## percent\",\n",
    "#                                                \"tokyo stocks close up # percent to fresh record high\")\n",
    "#\n",
    "#print(\"ROUGE-1: {}, ROUGE-2: {}, ROUGE-L: {}, ROUGE-BE: {}\".format(\n",
    "#        rouge_1, rouge_2, rouge_l, rouge_be\n",
    "#    ).replace(\", \", \"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EyKcd5ffKEBJ"
   },
   "outputs": [],
   "source": [
    "#https://pymotw.com/2/xml/etree/ElementTree/create.html\n",
    "\n",
    "bleu_arr = []\n",
    "rouge_1_arr  = []\n",
    "rouge_2_arr  = []\n",
    "rouge_L_arr  = []\n",
    "rouge_be_arr = []\n",
    "\n",
    "from xml.etree import ElementTree\n",
    "from xml.dom import minidom\n",
    "from functools import reduce\n",
    "\n",
    "def prettify(elem):\n",
    "    \"\"\"Return a pretty-printed XML string for the Element.\n",
    "    \"\"\"\n",
    "    rough_string = ElementTree.tostring(elem, 'utf-8')\n",
    "    reparsed = minidom.parseString(rough_string)\n",
    "    return reparsed.toprettyxml(indent=\"  \")\n",
    "  \n",
    "from xml.etree.ElementTree import Element, SubElement, Comment\n",
    "\n",
    "top = Element('ZakSum')\n",
    "\n",
    "comment = Comment('Generated by Amr Zaki')\n",
    "top.append(comment)\n",
    "\n",
    "i=0\n",
    "for summ in summary_array:\n",
    "  example = SubElement(top, 'example')\n",
    "  article_element   = SubElement(example, 'article')\n",
    "  article_element.text = article[i]\n",
    "  \n",
    "  reference_element = SubElement(example, 'reference')\n",
    "  reference_element.text = reference[i]\n",
    "  \n",
    "  summary_element   = SubElement(example, 'summary')\n",
    "  summary_element.text = summ\n",
    "\n",
    "  rouge_1, rouge_2,rouge_L,rouge_be,bleu_score = eval_rouges(reference[i],summ )\n",
    "  \n",
    "  eval_element = SubElement(example, 'eval')\n",
    "  bleu_score_element = SubElement(eval_element,'BLEU', {'score':str(bleu_score)})\n",
    "  ROUGE_1_element  = SubElement(eval_element, 'ROUGE_1' , {'score':str(rouge_1)})\n",
    "  ROUGE_2_element  = SubElement(eval_element, 'ROUGE_2' , {'score':str(rouge_2)})\n",
    "  ROUGE_L_element  = SubElement(eval_element, 'ROUGE_l' , {'score':str(rouge_L)})\n",
    "  ROUGE_be_element  = SubElement(eval_element,'ROUGE_be', {'score':str(rouge_be)})\n",
    "  \n",
    "  bleu_arr.append(bleu_score) \n",
    "  rouge_1_arr.append(rouge_1) \n",
    "  rouge_2_arr.append(rouge_2) \n",
    "  rouge_L_arr.append(rouge_L) \n",
    "  rouge_be_arr.append(rouge_be) \n",
    "\n",
    "  i+=1\n",
    "\n",
    "top.set('bleu', str(reduce(lambda x, y: x + y,  bleu_arr) / len(bleu_arr)))\n",
    "top.set('rouge_1', str(reduce(lambda x, y: x + y,  rouge_1_arr) / len(rouge_1_arr)))\n",
    "top.set('rouge_2', str(reduce(lambda x, y: x + y,  rouge_2_arr) / len(rouge_2_arr)))\n",
    "top.set('rouge_L', str(reduce(lambda x, y: x + y,  rouge_L_arr) / len(rouge_L_arr)))\n",
    "top.set('rouge_be', str(reduce(lambda x, y: x + y, rouge_be_arr) / len(rouge_be_arr)))\n",
    "\n",
    "with open(default_path + \"result_valid_29_10_2018_5_28pm.xml\", \"w+\") as f:\n",
    "  print(prettify(top), file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ihl49sb7bIRv"
   },
   "outputs": [],
   "source": [
    "len(summary_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FvnBdHIpb-wo"
   },
   "outputs": [],
   "source": [
    "sentence = \"The Lion Air flight JT 610 had been carrying 189 people, including three children, when it disappeared from radar during a short flight from Jakarta to Pangkal Pinang on the Indonesian island of Bangka, according to Basarnas, Indonesia's national search and rescue agency.\""
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "xO0UtYGfCuU-",
    "qI3AJND0mb0r",
    "TbbtwLe7njGS",
    "K-Kj9e5M9lBr"
   ],
   "include_colab_link": true,
   "name": "Copy of Model 2 seq2seq using lib",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
